<?php


namespace app\general\controller;


use app\general\model\SqArticle;
use app\general\model\SqArticleHot;
use app\general\model\SqArticleReadLog;
use app\general\model\SqArticleReadLogLevel;
use app\general\model\SqArticleTimeReadLog;
use app\general\model\SqArticleTimeReadLogLevel;
use phpspider\core\requests;
use phpspider\core\selector;
use think\Controller;
use think\Request;

/**
 * 文章定时爬取
 */
class Implant extends Controller
{
    protected $sq_article_hot;
    public function __construct(Request $request = null)
    {
        parent::__construct($request);
        $this->sq_article_hot = new SqArticleHot();
    }
    /**
     * 后台抓取
     */
    public function clicb(){
        set_time_limit(0);
        $article_type = $this->sq_article_hot->article_genre;
        $types=array_keys($article_type);
        $is_true=false;
        foreach ($types as $v){
            $type=$v;
            $typename=$article_type[$type];
            $where=[
                "article_genre"=>$type
            ];
            $info=$this->sq_article_hot->field("pn")->where($where)->order("id desc")->find();
            if($info){
                $where=[
                    "article_genre"=>$type,
                    "pn"=>$info["pn"]
                ];
                $count=$this->sq_article_hot->field("pn")->where($where)->count();
                if($count==10){
                    $pn=$info["pn"]+1;
                }else{
                    $pn=$info["pn"];
                }
            }else{
                $pn=1;
            }
            requests::$input_encoding = 'GB2312';
            requests::$output_encoding = 'GB2312';
            //requests::set_timeout(0);
            requests::set_useragent(array(
                "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/",
                "Opera/9.80 (Android 3.2.1; Linux; Opera Tablet/ADR-1109081720; U; ja) Presto/2.8.149 Version/11.10",
                "Mozilla/5.0 (Android; Linux armv7l; rv:9.0) Gecko/20111216 Firefox/9.0 Fennec/9.0"
            ));
            $url = "https://www.vipyl.com/article/".$type."/list_".$pn.".html";
            $html = requests::get($url);
            $selector = ".//ul[@id='artadd']/li/div[@class='rightnr']/b/a/text()";
            //获取标题
            $title = selector::select($html, $selector,'xpath');
            //获取图片
            $selector = ".//ul[@id='artadd']/li/div[@class='leftimg']/a/img/@src";
            $images=selector::select($html, $selector,'xpath');
            //获取id
            $selector = ".//ul[@id='artadd']/li/div[@class='rightnr']/b/a/@href";
            $ids=selector::select($html, $selector,'xpath');
            //获取内容简介
            // $selector = ".//ul[@id='artadd']/li/div[@class='rightnr']/span/text()";
            // $contents=selector::select($html, $selector,'xpath');
            //获取时间
            // $selector = ".//ul[@id='artadd']/li/div[@class='rightnr']/span/i/text()";
            // $times=selector::select($html, $selector,'xpath');
            $id = $this->recordTimers(5);
            if(empty($ids)){
                $message=$typename."已无文章";
            }else{
                $flag=$this->getContents($ids,$title,$images,$type,$typename,$pn);
                $message=$flag?$typename."爬取文章成功":"爬取文章失败";
            }
            $this->updateTimerType($id,$message);
            if($flag){
                $is_true=true;
            }
        }
        return $is_true?$this->success("爬取成功"):$this->error("爬取失败");

    }
    //获取某个文章的内容
    public function getContents($ids,$title,$images,$type,$typename,$pn){
        requests::$input_encoding = 'GB2312';
        requests::$output_encoding = 'GB2312';
        //标记一次爬取一次
        $is_one=true;
        //标记是否爬取成功
        $success=false;
        foreach ($ids as $k=>$v){
            $titles=$title[$k];
            $articleHot=new SqArticleHot();
            $where=[
                'title'=>$titles,
                'article_genre'=>array("gt",0)
            ];
            $info=$articleHot->where($where)->count();
            if($info<1&&$is_one){
                $url2 = "https://www.vipyl.com".$v;
                $html = requests::get($url2);
                //获取标题
                // $selector=".//div[@class='Article_title']/h1/text()";
                // $title=selector::select($html, $selector,'xpath');;
                //获取内容
                $selector="//div[contains(@id,'changeBodyFont')]";
                $content=selector::select($html, $selector,'xpath');
                //使用正则截图
                $content=str_replace("<div><script><![CDATA[bdunion('detail_xx_top');]]></script></div>","",$content);
                $content=str_replace("src=\"","src=\"https://www.vipyl.com",$content);
                $data=array(
                    "title"=>$title[$k],
                    "article_genre"=>intval($type),
                    "article_type"=>0,
                    "thumb"=>"https://www.vipyl.com".$images[$k],
                    "source"=>"网络",
                    "author"=>"雨露",
                    "content"=>$content,
                    "create_time"=>time(),
                    'pn'=>$pn
                );

                // $formdata[]=$data;
                $flag=$articleHot->insertGetId($data);
                if($flag){
                    $success=true;
                    $is_one=false;
                }
            }
        }
        return $success;
    }
    //记录定时器调用时间
    public function recordTimers($type)
    {
        $data = [
            'type' => $type,
            'create_time' => time(),
            'status' => 0
        ];
        return db('sq_record_timer')->insertGetId($data);
    }

    public function updateTimerType($id,$remark='')
    {
        db('sq_record_timer')->where('id', $id)->update(['status'=>1, 'remark'=>$remark]);
    }
    /**
     * 热点资讯抓取
     */
    public function getHotArt(){
        $res = '';
        set_time_limit(0);
        $types=array_keys($this->sq_article_hot->article_type);
      /*  $types=[$types[0],$types[1],$types[2]];
        var_dump($types);die;*/
        foreach ($types as $type){
            $article_type = $this->sq_article_hot->article_type;
            $contents = curl_qq_new($type,$article_type);
            if($contents){
                //去除重复标题
                $titles = i_array_column($contents,'title');
                $list = $this->sq_article_hot->where('title','in',$titles)->select();
                if(!empty($list)){
                    $data = collection($list)->toArray();
                    $titles = i_array_column($data,'title');
                    if(!empty($titles)){
                        foreach ($contents as $key =>$val){
                            if(in_array($val['title'],$titles)){
                                unset($contents[$key]);
                            }
                        }
                    }
                }
                $id = $this->recordTimers(5);
                $res = $this->sq_article_hot->saveAll($contents);
                $message=($res)?$this->sq_article_hot->article_type[$type]."抓取成功":$this->sq_article_hot->article_type[$type]."暂无最新数据";
                $this->updateTimerType($id,$message);

            }else{
                $id = $this->recordTimers(5);
                $message=$this->sq_article_hot->article_type[$type]."暂无最新数据";
                $this->updateTimerType($id,$message);
            }
        }
        return $res ? $this->success('抓取成功') : $this->error('抓取失败');
    }
    /**
     * 每俩个小时增加阅读量
     */
    public function addAppReadNum(){
        //获取今日时间戳
        $beginToday=mktime(0,0,0,date('m'),date('d'),date('Y'));
        //获取前三天的时间戳
        $thereDay=$beginToday-63*86400;
        $sixDay=$beginToday-6*86400;
        $id = $this->recordTimers(2);
        $articles=(new SqArticle())->whereBetween("create_time",[$thereDay,$beginToday])->select();
        foreach ($articles as $a){//male_num
            $num=rand(2,6);
            $total_user_count= SqArticleReadLog::where('art_id', $a['id'])->group('user_id')->count();
            if($total_user_count>10){
                (new SqArticle())->where("id",$a['id'])->setInc("male_num",$num);
                (new SqArticle())->where("id",$a['id'])->setInc("app_reader_number",$num);
            }
        }
        $articles=(new SqArticle())->whereBetween("create_time",[$sixDay,$thereDay])->select();
        foreach ($articles as $a){
            $total_user_count= SqArticleReadLog::where('art_id', $a['id'])->group('user_id')->count();
            $num=rand(0,2);
            if($total_user_count>10){
                (new SqArticle())->where("id",$a['id'])->setInc("app_reader_number",$num);
                (new SqArticle())->where("id",$a['id'])->setInc("male_num",$num);
            }
        }
        $res="增加app端阅读量成功";
        $this->updateTimerType($id,$res);
        return "增加成功";
    }
    /**
     * 后台添加阅读量，实现间隔增加
     */
    public function addReadNum(){
       //获取阅读记录
        $id2 = $this->recordTimers(11);
        $readLog=(new SqArticleTimeReadLog())->field("*")->where("create_time","<=",time())->select();
        if($readLog){
            foreach ($readLog as $log){
                $id=$log['id'];
                $read_log_data= [
                    'user_id' =>$log['user_id'] ,
                    'art_id' =>$log['art_id'],
                    'create_time' =>$log['create_time'] ,
                    'update_time' =>$log['update_time'] ,
                    'time' => $log['time'],
                    'is_robot' => $log['is_robot'],
                ];
                (new SqArticleReadLog())->insert($read_log_data);
                (new SqArticleTimeReadLog())->where("id",$id)->delete();
                (new SqArticle())->where("id",$log['art_id'])->setInc("reader_number",1);
                (new SqArticle())->where("id",$log['art_id'])->setInc("illict_num",1);
            }
        }
        $readLevel=(new SqArticleTimeReadLogLevel())->where("create_time","<=",time())->select();
        if($readLevel){
            foreach ($readLevel as $red){
                $id=$red['id'];
                $read_log_level_data = [
                    'user_id' => $red['user_id'],
                    'art_id' => $red['art_id'],
                    'level'=>$red['level'],
                    'create_time' => $red['create_time'],
                    'update_time' => $red['update_time'],
                    'parent_id' => $red['parent_id']
                ];
                (new SqArticleReadLogLevel())->save($read_log_level_data);
                (new SqArticleTimeReadLogLevel())->where("id",$id)->delete();
            }
        }
        $res="后台间隔增加阅读量任务执行";
        $this->updateTimerType($id2,$res);
        echo "完成";
    }
    /**
     * 热点资讯内容爬取
     */
    public function getHotArts(){
        $data=array(
            "18"=>"ent",
            "13"=>"world",
            "19"=>"fashion",
            "20"=>"games",
            "21"=>"finance_stock",
            "22"=>"edu",
            "23"=>"tech",
            "24"=>'emotion',
            "25"=>"health",
            "27"=>"food",
            "28"=>"history",
            "29"=>'society',
            "30"=>'kepu'
        );
        $message=array(
            "18"=>"娱乐",
            "13"=>"国际",
            "19"=>"时尚",
            "20"=>"游戏",
            "21"=>"证券",
            "22"=>"教育",
            "23"=>"科技",
            "24"=>"情感",
            "25"=>"健康",
            "27"=>"美食",
            "28"=>"历史",
            "29"=>"百态",
            "30"=>'科学',
        );
        $sqArticleHot=new SqArticleHot();
        $where['article_type']=array(
            "in",array(18,13,19,20,21,22)
        );
        $titles=$sqArticleHot->where($where)->order("id desc")->limit(100)->column("title");
        foreach($data as $k=>$d) {
            $url = 'https://pacaio.match.qq.com/xw/site?ext='.$d;
            $list=$this->https_request($url);
            $source="腾讯新闻";
            $name=$message[$k];
            $author="腾讯".$name;
            $data_temp=[];
            foreach($list as $l){
                if(in_array($l['title'],$titles)){
                    continue;
                }
                $html=article_change($l['vurl']);
                if($html){
                    $data_temp[] = [
                        'article_type'=> $k ,
                        'title'=> $l['title'],
                        'thumb'=> $l['img'],
                        'source'=> $source,
                        'author'=> $author,
                        'content'=> $html,
                        'create_time' => strtotime($l['publish_time'])
                    ];
                }
            }
            $id = $this->recordTimers(5);
            if(count($data_temp)>=1){
                $res = $sqArticleHot->saveAll($data_temp);
                $messages=($res)?$name."抓取成功":$name."暂无最新数据";
            }else{
                $messages=$name."暂无最新数据";
            }
            $this->updateTimerType($id,$messages);
        }


    }
    public static function https_request($curl, $data=null, $https=true, $method='post'){
            $ch = curl_init();//初始化
            $cookie = 'pgv_pvi=5534392320; pgv_pvid=88769584; RK=v6yBQWRZOj; ptcz=f1537b3cea5d122240be8736d36a87246e8ea23b1a066100ad23b58807ec5be5; pac_uid=0_5e1ebfe16a5e3; ptui_loginuin=1576112431@qq.com; pt_local_token=123456789; pgv_info=ssid=s6096595766';
            if(empty($user_agent)) $user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36';
            curl_setopt($ch, CURLOPT_USERAGENT,$user_agent);
            curl_setopt($ch, CURLOPT_URL, $curl);//设置访问的URL
            curl_setopt($ch, CURLOPT_HEADER, false);//设置不需要头信息
            curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);//只获取页面内容，但不输出
            curl_setopt($ch,CURLOPT_COOKIE,$cookie);
            if($https){
                curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);//不做服务器认证
                curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);//不做客户端认证
            }
            if($method == 'post'){
                curl_setopt($ch, CURLOPT_POST, true);//设置请求是POST方式
                curl_setopt($ch, CURLOPT_POSTFIELDS, $data);//设置POST请求的数据
            }
            $str = curl_exec($ch);//执行访问，返回结果
            curl_close($ch);//关闭curl，释放资源
            $result=json_decode($str,true);
            return $result['data'];
    }


}
